import logging
import logging.config

import os
import re
import time, sys
from collections import defaultdict

logging.config.fileConfig("logging.conf")
logger = logging.getLogger("MAIN_LOG")

def cleanDirs(dirList):
    """erases all of the output files for a clean regeneration"""
    logging.warn("Cleaning the directories")
    for dir in dirList:
        if not os.path.isdir(dir):
            return
        for file in os.listdir(dir):
            fullpath=os.path.join(dir, file)
            if os.path.isfile(fullpath):
                try:
                    os.remove(fullpath)
                    logging.info("Removing " + fullpath)
                except:
                    logging.error("Error trying to delete %s" % fullpath)

def safeFileExists(file):
    try:
        open(file, 'r')
        logging.debug("File %s exists" % file)
        return True
    except:
        logging.debug("File %s does not exist" % file)
        return False

def safeFileClose(file):
    if(not(file ==None)):
        try:
            file.close()
        except:
            logging.error("Could not close file %s", file)

def safeFileRemove(fileList):
    for file in fileList:
        if(not(file is None)):
            try:
                os.remove(file)
            except:
                logging.error("Error deleting file %s", file)

def getchunks(FILE, size=1024*1024):
    f = open(FILE)
    while 1:
        start = f.tell()
        f.seek(size, 1)
        s = f.readline()
        yield start, f.tell() - start
        if not s:
            break


def parseFile(FILE, regex):       
    matchesList = list()
    pat = re.compile(regex)

    count = defaultdict(int)
    for chunk in getchunks(FILE):
        for key, value in process(FILE, chunk, pat).items():
            count[key] += value
    
    for key in sorted(count, key=count.get)[:10]:
        matchesList.append(key)
        print("%40s = %s" % (key, count[key]))
    return matchesList


def process(file, chunk, pat):
    f = open(file)
    f.seek(chunk[0])
    d = defaultdict(int)
    for page in pat.findall(f.read(chunk[1])):
        d[page] += 1
    return d



if __name__ == "__main__":
    timer = time.time
    t0 = timer()

    tmp = parseFile("/data/STRING/COG.mappings.v8.2.txt", r'^(\S+.*COG0855)')
    print("time taken in secs: " , timer() - t0)
    print(tmp)


